# -*- coding: utf-8 -*-
import scrapy
from scrapy import Selector
import re

from moose_poetry_spider.items import MooseProxyIpItem

'''
爬去代理 ip
https://www.kuaidaili.com/free/
'''


class MooseProxyIpSpider(scrapy.Spider):
    name = 'MooseProxyIpSpider'

    allowed_domains = ['www.kuaidaili.com']

    start_urls = ['https://www.kuaidaili.com/free/']

    page_num = 1

    def parse(self, response):
        proxy_ip_item = MooseProxyIpItem()

        tr_list = response.xpath("//table[@class='table table-bordered table-striped']/tbody/tr").extract()
        for tr in tr_list:
            tr_selector = Selector(text=tr)

            ip = tr_selector.xpath("//td[1]/text()").extract_first()

            port = tr_selector.xpath("//td[2]/text()").extract_first()

            anonymity = tr_selector.xpath("//td[3]/text()").extract_first()

            http_type = tr_selector.xpath("//td[4]/text()").extract_first()

            location = tr_selector.xpath("//td[5]/text()").extract_first()

            response_speed_str = tr_selector.xpath("//td[6]/text()").extract_first()
            response_speed = int("".join(re.findall(r'\d', response_speed_str)))
            if response_speed <= 1:
                proxy_ip_item['ip'] = ip
                proxy_ip_item['port'] = port
                proxy_ip_item['anonymity'] = anonymity
                proxy_ip_item['http_type'] = http_type
                proxy_ip_item['location'] = location
                proxy_ip_item['response_speed'] = response_speed

            if len(proxy_ip_item) > 0:
                yield proxy_ip_item

        # 抓取下一页数据
        while self.page_num < 3779:
            self.page_num += 1
            yield scrapy.Request(str.format("https://www.kuaidaili.com/free/inha/{}/", self.page_num),
                                 callback=self.parse,
                                 dont_filter=True)
